@s-hirano-ist/s-scripts 1.26.8 → 4.0.1

package/README.md

@@ -2,6 +2,8 @@
 
 CLI scripts for s-private data operations.
 
+The npm artifact bundles the required internal workspace code and generated Prisma client. Consumers install only this CLI package; internal s-private packages are not separate npm dependencies. Command names remain unchanged.
+
 ## Installation
 
 ```sh

package/dist/chunk-6ADTNHCK.js

@@ -0,0 +1,85 @@
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+import {
+  makeExportedAt,
+  withTransactionRetry
+} from "./chunk-M275YZQP.js";
+
+// ../application/dist/batch/notes-batch-service.js
+var NotesBatchApplicationService = class {
+  commandRepository;
+  constructor(commandRepository) {
+    this.commandRepository = commandRepository;
+  }
+  /**
+   * Resets notes for a new batch export.
+   *
+   * @remarks
+   * Delegates to repository's resetStatus which performs two operations
+   * atomically within a single transaction.
+   */
+  async resetNotes(userId) {
+    return this.commandRepository.resetStatus(userId, makeExportedAt());
+  }
+  /**
+   * Reverts notes from LAST_UPDATED back to UNEXPORTED.
+   */
+  async revertNotes(userId) {
+    return this.commandRepository.bulkUpdateStatus({
+      userId,
+      fromStatus: "LAST_UPDATED",
+      toStatus: "UNEXPORTED"
+    });
+  }
+};
+
+// ../infrastructure/dist/batch/notes-command-repository.js
+function createNotesCommandRepository(prisma) {
+  return {
+    async bulkUpdateStatus(params) {
+      const { userId, fromStatus, toStatus, exportedAt } = params;
+      const result = await prisma.note.updateMany({
+        where: {
+          userId,
+          status: fromStatus
+        },
+        data: {
+          status: toStatus,
+          ...exportedAt && { exportedAt }
+        }
+      });
+      return { count: result.count };
+    },
+    async resetStatus(userId, exportedAt) {
+      const [finalized, marked] = await withTransactionRetry(() => prisma.$transaction([
+        prisma.note.updateMany({
+          where: {
+            userId,
+            status: "LAST_UPDATED"
+          },
+          data: {
+            status: "EXPORTED",
+            exportedAt
+          }
+        }),
+        prisma.note.updateMany({
+          where: {
+            userId,
+            status: "UNEXPORTED"
+          },
+          data: {
+            status: "LAST_UPDATED"
+          }
+        })
+      ]));
+      return {
+        finalized: { count: finalized.count },
+        marked: { count: marked.count }
+      };
+    }
+  };
+}
+
+export {
+  NotesBatchApplicationService,
+  createNotesCommandRepository
+};

package/dist/chunk-7PVU2Z32.js

@@ -0,0 +1,71 @@
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+
+// ../notification/dist/errors.js
+var NotificationError = class extends Error {
+  constructor(message = "notificationSend") {
+    super(message);
+    this.name = "NotificationError";
+  }
+};
+
+// ../notification/dist/pushover.js
+var PushoverService = class {
+  config;
+  constructor(config) {
+    this.config = config;
+  }
+  /**
+   * Sends a message to the Pushover API.
+   *
+   * @param message - The message to send
+   * @param priority - The priority level (-1 = low, 0 = normal, 1 = high)
+   *
+   * @internal
+   */
+  async sendMessage(message, priority = 0) {
+    try {
+      const body = new URLSearchParams({
+        token: this.config.appToken,
+        user: this.config.userKey,
+        message,
+        priority: String(priority)
+      });
+      const result = await fetch(this.config.url, {
+        method: "POST",
+        headers: { "Content-Type": "application/x-www-form-urlencoded" },
+        body
+      });
+      if (!result.ok) {
+        throw new NotificationError();
+      }
+    } catch (error) {
+      if (error instanceof NotificationError) {
+        console.error(`[NotificationService] Failed to send notification: ${error.message}`);
+      } else {
+        console.error("[NotificationService] Unknown error occurred while sending notification:", error);
+      }
+    }
+  }
+  async notifyError(message, context) {
+    const enrichedMessage = `[ERROR] ${context.caller}: ${message}`;
+    await this.sendMessage(enrichedMessage, 1);
+  }
+  async notifyWarning(message, context) {
+    const enrichedMessage = `[WARN] ${context.caller}: ${message}`;
+    await this.sendMessage(enrichedMessage, 0);
+  }
+  async notifyInfo(message, context) {
+    const enrichedMessage = `[INFO] ${context.caller}: ${message}`;
+    await this.sendMessage(enrichedMessage, -1);
+  }
+};
+function createPushoverService(config) {
+  return new PushoverService(config);
+}
+
+// ../notification/dist/web-push.js
+import webPush from "web-push";
+
+export {
+  createPushoverService
+};

package/dist/chunk-AJMTRCPJ.js

@@ -0,0 +1,15 @@
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+
+// src/lib/glob-paths.ts
+import { glob } from "node:fs/promises";
+async function globPaths(pattern, options = {}) {
+  const paths = [];
+  for await (const path of glob(pattern, options)) {
+    paths.push(path);
+  }
+  return paths.toSorted((left, right) => left.localeCompare(right, "en"));
+}
+
+export {
+  globPaths
+};

package/dist/chunk-AUD3PD6K.js

@@ -0,0 +1,85 @@
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+import {
+  makeExportedAt,
+  withTransactionRetry
+} from "./chunk-M275YZQP.js";
+
+// ../application/dist/batch/books-batch-service.js
+var BooksBatchApplicationService = class {
+  commandRepository;
+  constructor(commandRepository) {
+    this.commandRepository = commandRepository;
+  }
+  /**
+   * Resets books for a new batch export.
+   *
+   * @remarks
+   * Delegates to repository's resetStatus which performs two operations
+   * atomically within a single transaction.
+   */
+  async resetBooks(userId) {
+    return this.commandRepository.resetStatus(userId, makeExportedAt());
+  }
+  /**
+   * Reverts books from LAST_UPDATED back to UNEXPORTED.
+   */
+  async revertBooks(userId) {
+    return this.commandRepository.bulkUpdateStatus({
+      userId,
+      fromStatus: "LAST_UPDATED",
+      toStatus: "UNEXPORTED"
+    });
+  }
+};
+
+// ../infrastructure/dist/batch/books-command-repository.js
+function createBooksCommandRepository(prisma) {
+  return {
+    async bulkUpdateStatus(params) {
+      const { userId, fromStatus, toStatus, exportedAt } = params;
+      const result = await prisma.book.updateMany({
+        where: {
+          userId,
+          status: fromStatus
+        },
+        data: {
+          status: toStatus,
+          ...exportedAt && { exportedAt }
+        }
+      });
+      return { count: result.count };
+    },
+    async resetStatus(userId, exportedAt) {
+      const [finalized, marked] = await withTransactionRetry(() => prisma.$transaction([
+        prisma.book.updateMany({
+          where: {
+            userId,
+            status: "LAST_UPDATED"
+          },
+          data: {
+            status: "EXPORTED",
+            exportedAt
+          }
+        }),
+        prisma.book.updateMany({
+          where: {
+            userId,
+            status: "UNEXPORTED"
+          },
+          data: {
+            status: "LAST_UPDATED"
+          }
+        })
+      ]));
+      return {
+        finalized: { count: finalized.count },
+        marked: { count: marked.count }
+      };
+    }
+  };
+}
+
+export {
+  BooksBatchApplicationService,
+  createBooksCommandRepository
+};

package/dist/chunk-F4J6JBR4.js

@@ -0,0 +1,115 @@
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+
+// ../storage/dist/src/client.js
+import { Client } from "minio";
+
+// ../storage/dist/src/cf-access-transport.js
+import * as https from "node:https";
+function createCfAccessTransport(config) {
+  if (!config?.clientId || !config.clientSecret) {
+    return void 0;
+  }
+  return {
+    request(options, ...rest) {
+      if (typeof options === "object" && !(options instanceof URL)) {
+        const headers = options.headers ?? {};
+        headers["CF-Access-Client-Id"] = config.clientId;
+        headers["CF-Access-Client-Secret"] = config.clientSecret;
+        options.headers = headers;
+      }
+      const request2 = https.request;
+      return request2(options, ...rest);
+    }
+  };
+}
+
+// ../storage/dist/src/client.js
+function createMinioClient(config, cfAccess) {
+  const resolved = config ?? {
+    endPoint: process.env.MINIO_HOST ?? "",
+    port: Number(process.env.MINIO_PORT),
+    useSSL: process.env.MINIO_USE_SSL === void 0 ? !["localhost", "127.0.0.1"].includes(process.env.MINIO_HOST ?? "") : process.env.MINIO_USE_SSL === "true",
+    accessKey: process.env.MINIO_ACCESS_KEY ?? "",
+    secretKey: process.env.MINIO_SECRET_KEY ?? ""
+  };
+  const resolvedCfAccess = cfAccess ?? {
+    clientId: process.env.CF_ACCESS_CLIENT_ID ?? "",
+    clientSecret: process.env.CF_ACCESS_CLIENT_SECRET ?? ""
+  };
+  const transport = resolved.useSSL ? createCfAccessTransport(resolvedCfAccess) : void 0;
+  if (resolved.useSSL && !transport) {
+    throw new Error("CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET are required when useSSL is true.");
+  }
+  return new Client({
+    endPoint: resolved.endPoint,
+    port: resolved.port,
+    useSSL: resolved.useSSL,
+    accessKey: resolved.accessKey,
+    secretKey: resolved.secretKey,
+    ...transport && { transport }
+  });
+}
+
+// ../storage/dist/index.js
+import { S3Error } from "minio";
+
+// ../storage/dist/src/errors.js
+var StorageOperationError = class extends Error {
+  context;
+  constructor(context, cause) {
+    super(`Storage ${context.operation} failed for ${context.objectKey}`, {
+      cause
+    });
+    this.name = "StorageOperationError";
+    this.context = context;
+  }
+};
+
+// ../storage/dist/src/storage-service.js
+function createStorageService(client, bucketName, pathConfig) {
+  function buildObjectKey(path, isThumbnail) {
+    return `${isThumbnail ? pathConfig.thumbnailPrefix : pathConfig.originalPrefix}/${path}`;
+  }
+  function wrapStorageError(operation, objectKey, isThumbnail, error) {
+    throw new StorageOperationError({ operation, objectKey, bucketName, isThumbnail }, error);
+  }
+  return {
+    async uploadImage(path, bytes, isThumbnail) {
+      const objKey = buildObjectKey(path, isThumbnail);
+      try {
+        await client.putObject(bucketName, objKey, Buffer.from(bytes));
+      } catch (error) {
+        wrapStorageError("uploadImage", objKey, isThumbnail, error);
+      }
+    },
+    async getImage(path, isThumbnail) {
+      const objKey = buildObjectKey(path, isThumbnail);
+      try {
+        return await client.getObject(bucketName, objKey);
+      } catch (error) {
+        return wrapStorageError("getImage", objKey, isThumbnail, error);
+      }
+    },
+    async getImageOrThrow(path, isThumbnail) {
+      const objKey = buildObjectKey(path, isThumbnail);
+      try {
+        await client.statObject(bucketName, objKey);
+      } catch (error) {
+        wrapStorageError("getImageOrThrow", objKey, isThumbnail, error);
+      }
+    },
+    async deleteImage(path, isThumbnail) {
+      const objKey = buildObjectKey(path, isThumbnail);
+      try {
+        await client.removeObject(bucketName, objKey);
+      } catch (error) {
+        wrapStorageError("deleteImage", objKey, isThumbnail, error);
+      }
+    }
+  };
+}
+
+export {
+  createMinioClient,
+  createStorageService
+};

package/dist/chunk-HDYQV2Y2.js

@@ -0,0 +1,58 @@
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+
+// ../search/dist/frontmatter.js
+import { load } from "js-yaml";
+var BYTE_ORDER_MARK = "\uFEFF";
+var OPENING_DELIMITER_REGEX = /^---[\t ]*\r?\n/u;
+var CLOSING_DELIMITER_REGEX = /^---[\t ]*\r?$/gmu;
+function stripByteOrderMark(source) {
+  return source.startsWith(BYTE_ORDER_MARK) ? source.slice(1) : source;
+}
+function getContentStart(source, delimiterEnd) {
+  if (source.startsWith("\r\n", delimiterEnd)) {
+    return delimiterEnd + 2;
+  }
+  if (source.startsWith("\n", delimiterEnd)) {
+    return delimiterEnd + 1;
+  }
+  return delimiterEnd;
+}
+function parseYamlObject(source) {
+  if (source.trim() === "") {
+    return {};
+  }
+  const parsed = load(source);
+  if (parsed === void 0) {
+    return {};
+  }
+  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+    throw new Error("Markdown frontmatter must be a YAML object.");
+  }
+  return parsed;
+}
+function parseMarkdownFrontmatter(source) {
+  const normalizedSource = stripByteOrderMark(source);
+  const openingDelimiter = OPENING_DELIMITER_REGEX.exec(normalizedSource);
+  if (!openingDelimiter) {
+    if (/^---[\t ]*$/u.test(normalizedSource)) {
+      throw new Error("Markdown frontmatter is missing a closing delimiter.");
+    }
+    return { content: normalizedSource, data: {} };
+  }
+  CLOSING_DELIMITER_REGEX.lastIndex = openingDelimiter[0].length;
+  const closingDelimiter = CLOSING_DELIMITER_REGEX.exec(normalizedSource);
+  CLOSING_DELIMITER_REGEX.lastIndex = 0;
+  if (!closingDelimiter) {
+    throw new Error("Markdown frontmatter is missing a closing delimiter.");
+  }
+  const yamlSource = normalizedSource.slice(openingDelimiter[0].length, closingDelimiter.index);
+  const contentStart = getContentStart(normalizedSource, closingDelimiter.index + closingDelimiter[0].length);
+  return {
+    content: normalizedSource.slice(contentStart),
+    data: parseYamlObject(yamlSource)
+  };
+}
+
+export {
+  parseMarkdownFrontmatter
+};

package/dist/chunk-J4VLH4VA.js

@@ -0,0 +1,267 @@
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+
+// ../search/dist/qdrant-client.js
+import { QdrantClient } from "@qdrant/js-client-rest";
+
+// ../search/dist/config.js
+function requireEnv(name) {
+  const value = process.env[name];
+  if (!value)
+    throw new Error(`${name} environment variable is required`);
+  return value;
+}
+function envWithDefault(name, defaultValue) {
+  return process.env[name] || defaultValue;
+}
+var RAG_CONFIG = {
+  qdrant: {
+    collectionName: requireEnv("QDRANT_COLLECTION_NAME"),
+    vectorSize: Number(envWithDefault("EMBEDDING_VECTOR_SIZE", "384")),
+    distance: "Cosine"
+  },
+  embedding: {
+    // Deployed services and ingestion jobs always use Qdrant Cloud Inference.
+    // Keep TEI only for explicitly enabled, non-production local development.
+    url: process.env.LOCAL_DEV_MODE === "true" && process.env.VERCEL !== "1" && process.env.NODE_ENV !== "production" ? process.env.EMBEDDING_URL : void 0,
+    model: envWithDefault("EMBEDDING_MODEL", "intfloat/multilingual-e5-small")
+  },
+  chunking: { maxChunkLength: 2e3, headingLevels: [2, 3] }
+};
+
+// ../search/dist/qdrant-client.js
+var client = null;
+async function embedLocally(texts) {
+  const { url } = RAG_CONFIG.embedding;
+  if (!url)
+    throw new Error("EMBEDDING_URL environment variable is required");
+  const response = await fetch(`${url.replace(/\/$/u, "")}/embed`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ inputs: texts, normalize: true, truncate: true })
+  });
+  if (!response.ok) {
+    throw new Error(`Embedding request failed (${response.status}): ${await response.text()}`);
+  }
+  const embeddings = await response.json();
+  if (!Array.isArray(embeddings) || embeddings.length !== texts.length || embeddings.some((embedding) => !Array.isArray(embedding) || embedding.some((value) => typeof value !== "number"))) {
+    throw new TypeError("Embedding API returned an invalid response");
+  }
+  return embeddings;
+}
+function getQdrantClient(config) {
+  if (!client) {
+    const url = config?.url ?? process.env.QDRANT_URL;
+    const apiKey = config?.apiKey ?? process.env.QDRANT_API_KEY;
+    if (!url) {
+      throw new Error("QDRANT_URL environment variable is required");
+    }
+    client = new QdrantClient({
+      url,
+      apiKey
+    });
+  }
+  return client;
+}
+async function deleteCollection(collectionName) {
+  const qdrant = getQdrantClient();
+  await qdrant.deleteCollection(collectionName);
+}
+async function ensureCollection() {
+  const qdrant = getQdrantClient();
+  const { collectionName, vectorSize, distance } = RAG_CONFIG.qdrant;
+  const collections = await qdrant.getCollections();
+  const exists = collections.collections.some((c) => c.name === collectionName);
+  if (exists) {
+    console.log(`Collection ${collectionName} already exists.`);
+  } else {
+    console.log(`Creating collection: ${collectionName}`);
+    await qdrant.createCollection(collectionName, {
+      vectors: {
+        size: vectorSize,
+        distance
+      }
+    });
+    console.log(`Collection ${collectionName} created successfully.`);
+  }
+  await qdrant.createPayloadIndex(collectionName, {
+    field_name: "type",
+    field_schema: "keyword",
+    wait: true
+  });
+  await qdrant.createPayloadIndex(collectionName, {
+    field_name: "top_heading",
+    field_schema: "keyword",
+    wait: true
+  });
+  await qdrant.createPayloadIndex(collectionName, {
+    field_name: "content_type",
+    field_schema: "keyword",
+    wait: true
+  });
+  console.log("Payload indexes ensured for: type, top_heading, content_type");
+}
+async function upsertPoints(points) {
+  const qdrant = getQdrantClient();
+  const { collectionName } = RAG_CONFIG.qdrant;
+  const localEmbeddings = RAG_CONFIG.embedding.url ? await embedLocally(points.map((point) => point.text)) : void 0;
+  const qdrantPoints = points.map((p, index) => ({
+    id: hashToUint(p.id),
+    vector: localEmbeddings?.[index] ?? {
+      text: p.text,
+      model: RAG_CONFIG.embedding.model
+    },
+    payload: p.payload
+  }));
+  await qdrant.upsert(collectionName, {
+    wait: true,
+    points: qdrantPoints
+  });
+}
+async function getExistingHashes(chunkIds) {
+  const qdrant = getQdrantClient();
+  const { collectionName } = RAG_CONFIG.qdrant;
+  const hashMap = /* @__PURE__ */ new Map();
+  if (chunkIds.length === 0)
+    return hashMap;
+  const numericIds = chunkIds.map((id) => hashToUint(id));
+  try {
+    const result = await qdrant.retrieve(collectionName, {
+      ids: numericIds,
+      with_payload: ["chunk_id", "content_hash"]
+    });
+    for (const point of result) {
+      const payload = point.payload;
+      if (payload?.chunk_id && payload?.content_hash) {
+        hashMap.set(payload.chunk_id, payload.content_hash);
+      }
+    }
+  } catch {
+  }
+  return hashMap;
+}
+async function search(queryText, options = {}) {
+  const qdrant = getQdrantClient();
+  const { collectionName } = RAG_CONFIG.qdrant;
+  const { topK = 10, filter } = options;
+  const filterConditions = [];
+  if (filter?.type) {
+    filterConditions.push({
+      key: "type",
+      match: { value: filter.type }
+    });
+  }
+  if (filter?.top_heading) {
+    filterConditions.push({
+      key: "top_heading",
+      match: { value: filter.top_heading }
+    });
+  }
+  if (filter?.content_type) {
+    if (Array.isArray(filter.content_type)) {
+      filterConditions.push({
+        key: "content_type",
+        match: { any: filter.content_type }
+      });
+    } else {
+      filterConditions.push({
+        key: "content_type",
+        match: { value: filter.content_type }
+      });
+    }
+  }
+  const query = RAG_CONFIG.embedding.url ? (await embedLocally([queryText]))[0] : { text: queryText, model: RAG_CONFIG.embedding.model };
+  const result = await qdrant.query(collectionName, {
+    query,
+    limit: topK,
+    with_payload: true,
+    filter: filterConditions.length > 0 ? { must: filterConditions } : void 0
+  });
+  return result.points.map((r) => {
+    const payload = r.payload;
+    return {
+      score: r.score,
+      text: payload.text,
+      title: payload.title,
+      url: payload.url,
+      heading_path: payload.heading_path,
+      type: payload.type,
+      content_type: payload.content_type,
+      doc_id: payload.doc_id
+    };
+  });
+}
+async function listAllChunkIds() {
+  const qdrant = getQdrantClient();
+  const { collectionName } = RAG_CONFIG.qdrant;
+  const chunkIds = [];
+  const pageSize = 256;
+  let nextOffset;
+  try {
+    do {
+      const result = await qdrant.scroll(collectionName, {
+        limit: pageSize,
+        offset: typeof nextOffset === "string" || typeof nextOffset === "number" ? nextOffset : void 0,
+        with_payload: ["chunk_id"],
+        with_vector: false
+      });
+      for (const point of result.points) {
+        const payload = point.payload;
+        if (payload?.chunk_id) {
+          chunkIds.push(payload.chunk_id);
+        }
+      }
+      nextOffset = result.next_page_offset;
+    } while (nextOffset !== null && nextOffset !== void 0);
+  } catch {
+  }
+  return chunkIds;
+}
+async function deletePointsByChunkIds(chunkIds) {
+  if (chunkIds.length === 0)
+    return;
+  const qdrant = getQdrantClient();
+  const { collectionName } = RAG_CONFIG.qdrant;
+  const numericIds = chunkIds.map((id) => hashToUint(id));
+  await qdrant.delete(collectionName, {
+    wait: true,
+    points: numericIds
+  });
+}
+async function getCollectionStats() {
+  const qdrant = getQdrantClient();
+  const { collectionName } = RAG_CONFIG.qdrant;
+  try {
+    const info = await qdrant.getCollection(collectionName);
+    return {
+      pointsCount: info.points_count ?? 0,
+      status: info.status
+    };
+  } catch {
+    return {
+      pointsCount: 0,
+      status: "not_found"
+    };
+  }
+}
+function hashToUint(str) {
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    const char = str.codePointAt(i) ?? 0;
+    hash = (hash << 5) - hash + char;
+    hash = hash & hash;
+  }
+  return Math.abs(hash);
+}
+
+export {
+  RAG_CONFIG,
+  getQdrantClient,
+  deleteCollection,
+  ensureCollection,
+  upsertPoints,
+  getExistingHashes,
+  search,
+  listAllChunkIds,
+  deletePointsByChunkIds,
+  getCollectionStats
+};